
from collections import deque

import datetime
import threading
from UpdateQzone_crawler import UpdateBot
from time import ctime, sleep
import os
import sys
sys.path.append("..")
import lib 
from lib.utils import Logger, DummyLogger, transcode5, timeago
from lib import UpdateQzone_crawler 
from lib.database_qzone import Database
try:
    import cPickle as pickle
except:
    import pickle
from cStringIO import StringIO
import traceback, string


def savefile(name,obj):      
          
        file = open(name, 'w')
        if(file):
                pickle.dump(obj, file)          # pickle to file
                file.close( )                     # any file-like object will do

def loadfile(name):
      
        if(os.path.isfile(name)):
            file = open(name, 'r')
            if(file):
                obj = pickle.load(file)         # unpickle from file
                file.close( )                     # re-creates object in memory
                return obj
            else:
                return 0
        else:
            return 0
class DBUpdater():

       def __init__(self, db_engine = None):
         self.db=Database()
         self.userlist=[] #element: (1L, '100010', 22L, False, datetime.datetime(2010, 1, 26, 22, 24, 53), 0L)
         self.newuserlist=[]
         self.maxtime=datetime.datetime(1970, 1, 1)
         self.maxseq=0
         self.uid=0
         self.lastuid=0
         self.lastblogseq=0
         self.newuserintervallist=[]
         self.userintervallist=[]
         self.today=datetime.datetime.now( )
         self.validday=self.today-datetime.timedelta(days=365)
         self.updatePolicy=(7,30,60)
         listsize=len(self.updatePolicy)+2
         self.Updatelist=[]
         self.curuserp=0
         self.curblogp=0
         self.dir="./DBUpdater/"
         self.uidhash={}
         for i in range(0,listsize):
            self.Updatelist.append(deque([]))
         print self.validday,self.today
        
         self.Loadlast()
       
         #self.hash={'seq':0,'id':1,'articlenum':2,'done':3,'modtime':4,'numclasses':5}
     

       def GetnewUsers(self):
         #self.maxtime=loadfile("lastmodtime")
         self.newuserlist=[]
         print "reloading new users------------------------------"
       
         i=0
         tempnewlist=self.db._queryNewUser(self.maxseq,i)
         i+=1
         print "get ",len(tempnewlist),"'s users from DB."
         #self.newuserlist.extend(tempnewlist)
         for user in tempnewlist:
           self.newuserlist.append((user[4],user[0],user[1],user[2],user[3]))
         while(tempnewlist):
            tempnewlist=self.db._queryNewUser(self.maxseq,i)
            i+=1
            print "get ",len(tempnewlist),"'s userifos from DB"
            #self.newuserlist.extend(newlist)
            for user in tempnewlist:
              self.newuserlist.append((user[4],user[0],user[1],user[2],user[3]))
         self.newuserlist.sort(key=lambda x:x[1])
         print "New user list size:",len(self.newuserlist)
     
       def AddtoUserlist(self):
         #userlist=self.db._queryUser(0)
       
       
         #print userlist
         self.userlist.extend(self.newuserlist)
      
         savefile(self.dir+"userlist",self.userlist)  
         print len(self.userlist)

    

       def GetMaxModtime(self,li):
         for user in li:
           if (self.maxtime<=user[4]):
                self.maxtime=user[4]
                self.uid=user[1]   
           if(self.maxseq<user[0]):
                self.maxseq=user[0]
         #print "maxtime"," maxseq"," maxuid"     
         #print self.maxtime,self.maxseq,self.uid
      
       def savepresent(self):
         savefile(self.dir+"lastmodtime",[self.maxtime,self.maxseq,self.blogseq])  
         self.saveUpdatelist()
         self.lastblogseq=self.blogseq

       def Loadlast(self):
         try:
            self.maxtime,self.maxseq,self.lastblogseq=loadfile(self.dir+"lastmodtime")
            print "Load from lastmodtime,lastmodtime:",self.maxtime," lastblogseq:",self.lastblogseq
     
         except Exception,e:
            print "maxtime"," maxseq"," lastblogseq"     
            print self.maxtime,self.maxseq,self.lastblogseq
            self.maxtime,self.maxseq,self.lastblogseq=[datetime.datetime(1970, 1, 1),0,0]
            pass
         self.loadUpdatelist()
    

       def GuessInterval(self,uid):
          #binfos=self.db._queryUserBlogInfo(str(uid))# (datetime.datetime(2009, 3, 10, 0, 0), 1L, 114L),posttime,numofcomments,numofreads)
          binfos=[]
          num_blogs=0
          while(self.newbloglist[self.curblogp][3]==uid):
                     num_blogs+=1
                     binfos.append(self.newbloglist[self.curblogp])
                     self.curblogp+=1
                     if(self.curblogp>=len(self.newbloglist)):
                         break
                       
          #print uid,len(binfos)
          if(not binfos):
               print uid," not found"
               return   -1,-1   
         
          binfos.sort(key=lambda x:x[0])
         
          #print binfos
          IntervalList=[]
          date=self.validday
          count=0
          intervalgap=0
          for info in binfos:
              if(info[0]>date):               
                 IntervalList.append((intervalgap,date-self.validday,count))
                 intervalgap=info[0]-date
                 date=info[0]
                 count=1
              elif(info[0]==date):
                 count+=1 
              else:
                 #print "error"
                 pass
          IntervalList.append((intervalgap,date-self.validday,count))
          IntervalList.append((self.today-binfos[-1][2],binfos[-1][2]-self.validday,count))
          inttotal=0#datetime.timedelta(days=0)
          intertotal=0
          del IntervalList[0:1]
          #print "The list is:"
          #print IntervalList
          for interval in IntervalList:
              intinterval=int(interval[0].days)
              days=int(interval[1].days)
              count=interval[2]
              #print intinterval,days,count
           
              inttotal+=intinterval*days*count
              intertotal+=days*count
          try:
            timeInterval=float(inttotal)/float(intertotal)
          except Exception,e:
            timeInterval=365 
          return int(timeInterval),num_blogs
     
       def GuessUserInterval(self):
          
           self.curblogp=0
           print "GuessUserInterval newuserlist length:",len(self.newuserlist)
           i=self.curuserp
           #self.uidhash={}
           for self.curuserp in range(0,len(self.newuserlist)):
                    if(self.curblogp>=len(self.newbloglist)):
                         break
                   
                    user=self.newuserlist[self.curuserp]
                    if(user[1]<self.newbloglist[self.curblogp][3]):
                         continue
                    interval,num_blogs=self.GuessInterval(user[1])
                    #print user[1],interval
                    if(not interval==-1):
                     if(self.uidhash.has_key(user[1])):
                       oldinterval=self.uidhash[user[1]][5]
                       interval=int(float(oldinterval)*float(interval)/(float(oldinterval)+float(interval)))
                       self.uidhash[user[1]][5]=interval
                       self.uidhash[user[1]][6]+=num_blogs
                       #self.newuserintervallist.append(self.uidhash[user[1]]) 
                       #print user[1],"num_blogs",self.uidhash[user[1]][6]
                     else:
                       temp=[user[0],user[1],user[2],user[3],user[4],interval,num_blogs]
                       self.newuserintervallist.append(temp)   #{'seq':0,'id':1,'articlenum':2,'done':3,'modtime':4,'interval':5,'num_blogs':6}  
                       self.uidhash[user[1]]=temp                                           
           print len(self.newuserlist),":",self.curuserp," ",len(self.newbloglist),":",self.curblogp
          
           #print self.newuserlist[self.curuserp-1],self.newbloglist[self.curblogp-1][3]

       def GetUserblogs(self,userlist=None):
         #print "uid",self.uid
         if(0):
		 bloglist=self.db._queryUserBlogInfo(self.uid)
		 self.blogseq=self.lastblogseq
		 for blog in bloglist:
		   if (self.blogseq<=blog[3]):
		        self.blogseq=blog[3]
         self.blogseq=self.db._queryMaxBlogSeq()     
         print "Max blogseq",self.blogseq
         #print  "length:",len(bloglist)
         #print bloglist[1]
       
         step=1024*64
         start=int(self.lastblogseq)/step
         end=int(self.blogseq)/step
         be=0
         sp=0
         self.newuserintervallist=[]
         for i in range(start,end+1): 
               print "--------------------------------"
                
               if(i>start and i<end):
                  sp=(i+1)*step
                  be=i*step
               elif(i==start):
                  be=int(self.lastblogseq)
                  sp=(i+1)*step
                  if(sp>int(self.blogseq)):
                      sp=int(self.blogseq)
               elif(i==end):
                  be=end*step
                  if(be<int(self.lastblogseq)):
                      be=int(self.lastblogseq)
                  sp=int(self.blogseq)
               print "deal with blogs from ",be," to ",sp    
               self.GetSomeUserblogs(be,sp)
               self.GuessUserInterval()
         self.dispachlist(self.newuserintervallist)
                                               
       def GetSomeUserblogs(self,begine,end):
        
       
         tempnewlist=[]
         self.newbloglist=[]
         i=0

         tempnewlist=self.db._queryNewUserBlogInfo(begine,end,i)
         i+=1
         print "get ",len(tempnewlist),"'s blogifos from DB."
       
         for bloginfo in tempnewlist:
           self.newbloglist.append((bloginfo[0],bloginfo[1],bloginfo[2],bloginfo[3]))
         while(tempnewlist):
            tempnewlist=self.db._queryNewUserBlogInfo(begine,end,i)
            i+=1
            print "get ",len(tempnewlist),"'s bloginfos from DB"
          
            for bloginfo in tempnewlist:
              self.newbloglist.append((bloginfo[0],bloginfo[1],bloginfo[2],bloginfo[3]))# 0:time 1:num_comment 2:num_read 3:authorid
      
         if (0):  
       
               print "try get blog infos from DB be:",be," sp:",sp
               #templist=self.db._queryNewUserBlogInfo(self.lastblogseq,self.blogseq)
               templist=self.db._queryNewUserBlogInfo(be,sp)
               for bloginfo in templist:
                  self.orgbloglist.append((bloginfo[0],bloginfo[1],bloginfo[2],bloginfo[3]))
       
      
         self.newbloglist.sort(key=lambda x:x[3])
      
         print "newbloglist length:",len(self.newbloglist)
      

    
 

       def dispachlist(self,intervallist):
            listsize=len(self.updatePolicy)+2
            print "intervallist:",len(intervallist)
            for interval in intervallist:
                if(not interval[3]):
                    self.Updatelist[0].append(interval)
                    continue
                flag=False
                for i in range(0,listsize-2):
                   if (interval[5]<=self.updatePolicy[i]):
                         self.Updatelist[i+1].append(interval)
                         flag=True
                         break
                if(not flag):
                   self.Updatelist[listsize-1].append(interval)
            length=0
            for i in range(0,len(self.Updatelist)):
               print "list ",i,": length:",len(self.Updatelist[i])
               length+=len(self.Updatelist[i])
            print "Total length:",length
     
       def saveUpdatelist(self):
           savefile(self.dir+"Updatelist",self.Updatelist)

       def loadUpdatelist(self):
           listsize=len(self.updatePolicy)+2
           self.Updatelist=[]
           try:
             print "loading Updatelist..."
             self.Updatelist=loadfile(self.dir+"Updatelist")
             length=0
             for i in range(0,len(self.Updatelist)):
               print "list ",i,": length:",len(self.Updatelist[i])
               length+=len(self.Updatelist[i])
             print "Total length:",length
           except Exception,e:
             print "load failed."
             pass
           if(not self.Updatelist):
              self.Updatelist=[]
              for i in range(0,listsize):
               self.Updatelist.append(deque([]))
         
       def ProcessNewlist(self):
           self.GetnewUsers()
           self.GetMaxModtime(self.newuserlist)
           self.GetUserblogs()
           self.savepresent()
           
         
       def startup(self):
          TimeInterval=(3600*24*2,3600*24*2,3600*24*7,3600*24*30,3600*24*60)
          Threads=(8,4,4,4,4)
          logger = Logger("./DBUpdater/DBUpdater.log",True)
          logger.log("start updating","Main")
          for i in range(0,len(self.Updatelist)):
                uc=updatecrawlermanager(i,self.db,self.Updatelist,Threads[i],TimeInterval[i],self.updatePolicy)
                uc.start()
                pass#UC=UpdateCrawler(self.Updatelist[i],self.db,$) #1 "updated" up 1  2 'No_New_post' down 1 3  "Not_finished" change to first 'blank' 'No_This_user' change to last

class updatecrawlermanager(threading.Thread):      
     def __init__(self, queue_no,db ,queue,threads,timeinterval,updatePolicy):
          self.db=db
          self.q=queue[queue_no]
          self.queuelist=queue
          self.queue_no=queue_no
          self.threads=threads
          self.timeinterval=timeinterval
          self.updatePolicy=updatePolicy
          self.name="updatecrawlermanager"+str(queue_no)
          threading.Thread.__init__(self, name = self.name)
 
     def run(self):
         #while(True):
           t=0
           logger = Logger("./DBUpdater/DBUpdater.log",True) 
           logger.log("start updating",self.name)
           try:
              bots_semaphore = threading.Semaphore(self.threads)
	      #a=[1,2,345,5,6,7,8,8,9]
	      userlist=list(self.q)
              length=len(userlist)
              for user in userlist:
                uid=user[1]
                num_blogs=user[2]
                
                #logger.log("user:"+str(user[1])+" num_blogs:"+str(num_blogs) , self.name)
                Ubot=UpdateBot(uid,num_blogs,logger,self.db,self.queue_no,self.q,user,self.queuelist,bots_semaphore,self.updatePolicy)# after crwaled do sth
                bots_semaphore.acquire()
                Ubot.start()
                t=t+1
                global countlist
                countlist[self.queue_no]=t
                toatal=0
                for c in countlist:
                    toatal+=c
                print countlist,toatal
                logger.log("length:"+str(length)+" positon:"+str(t),self.name)
              length=0
              
              for i in range(0,len(self.queuelist)):
               print "list ",i,": length:",len(self.queuelist[i])
               length+=len(self.queuelist[i])
              print "Total length:",length
              logger.log("sleeping "+str(self.timeinterval),self.name)
              logger.log("sleeping "+str(self.timeinterval),self.name)
              logger.log("sleeping "+str(self.timeinterval),self.name)
              sleep(self.timeinterval)
           except Exception,e:
              sio = StringIO()
              sio = StringIO()
              traceback.print_exc(file=sio)
              logger.warn('updatecrawlermanager throw exception abnormally', self.name)
              logger.warn(sio.getvalue(), self.name)
              logger.warn(str(e), self.name)
              sio.close()
  
 #self.db self.logger self.uid  self.user self.qs self.bots_semaphore

countlist=[0,0,0,0,0,0]
              
def test():
   dbu=DBUpdater()
   #dbu.LoadPickle()
   dbu.ProcessNewlist()
   dbu.startup()
   #dbu.GetMaxModtime(dbu.userlist)
   #dbu.GuessUserInterval()
   #dbu.GuessUserInterval()
   #dbu.GetMaxModtime()
   #dbu.Loadlastmodtime()
   #dbu.GetnewUsers()
   while(True):
      sleep(1000)
      dbu.ProcessNewlist()
 
test()

